## creates tables for appendix I:
## results for white respondents only
## figure A10 and table A9

rm(list=ls())

library(tidyverse)
library(kableExtra)


# read in data

opinion_bootstrapped_whites <- readRDS('bootstrap_gaps_data/opinion_bootstrapped_noleaners_whites.RDS')

for (i in names(opinion_bootstrapped_whites)) {
  assign(paste0(i, '_whites'), opinion_bootstrapped_whites[[i]])
}

# create polarization matrices

dem_polarization_boot_whites <- abs(dem_inctop_10_whites - dem_incbottom_10_whites)
rep_polarization_boot_whites <- abs(rep_inctop_10_whites - rep_incbottom_10_whites)
all_polarization_boot_whites <- abs(all_inctop_10_whites - all_incbottom_10_whites)



# table A9 - polarization details white only

polarization_tab_whites <- data.frame(expand.grid(party = c('rep', 'dem', 'all'),
                                                  topic_6 = unique(issuetopics_whites$topic_6),
                                                  stringsAsFactors = FALSE),
                                      avgpol = NA,
                                      signif = NA,
                                      gilens = NA,
                                      gilenssignif = NA,
                                      disagree = NA,
                                      disagreegilens = NA,
                                      n_quest = NA,
                                      stringsAsFactors = FALSE)

for (i in 1:nrow(polarization_tab_whites)) {
  temp.issues <- issuetopics_whites$question[issuetopics_whites$topic_6 == polarization_tab_whites$topic_6[i]]
  temp.rich <- get(paste0(polarization_tab_whites$party[i], '_inctop_10_whites'))
  temp.rich <- temp.rich[rownames(temp.rich) %in% temp.issues,]
  temp.poor <- get(paste0(polarization_tab_whites$party[i], '_incbottom_10_whites'))
  temp.poor <- temp.poor[rownames(temp.poor) %in% temp.issues,]
  temp.pol <- get(paste0(polarization_tab_whites$party[i], '_polarization_boot_whites'))
  temp.pol <- temp.pol[rownames(temp.pol) %in% temp.issues,]
  
  polarization_tab_whites$avgpol[i] <- median(colMeans(temp.pol))
  
  temp.ci <- apply(temp.rich - temp.poor, MARGIN = 1, function(x) quantile(x, c(0.025, 0.975)))
  polarization_tab_whites$signif[i] <- sum(!apply(temp.ci, MARGIN = 2, function(x) between(0, x[1], x[2])))
  
  polarization_tab_whites$gilens[i] <- sum(apply(temp.pol, MARGIN = 1, median) > 0.1)
  
  temp.ci <- apply(temp.pol, MARGIN = 1, function(x) quantile(x, c(0.025, 0.975)))
  polarization_tab_whites$gilenssignif[i] <- sum(apply(temp.ci, MARGIN = 2, function(x) all(x > 0.1) | all(x < -0.1)))
  
  temp.10pt <- apply(temp.ci, MARGIN = 2, function(x) all(x > 0.1) | all(x < -0.1))
  
  temp.disagree <- abs(apply((temp.rich > 0.5) - (temp.poor > 0.5), MARGIN =  1, median))
  
  polarization_tab_whites$disagree[i] <- sum(temp.disagree)
  
  polarization_tab_whites$disagreegilens[i] <- sum(temp.disagree*as.numeric(temp.10pt))
  
  polarization_tab_whites$n_quest[i] <- length(temp.issues)
}

for (p in unique(polarization_tab_whites$party)) {
  i <- nrow(polarization_tab_whites) + 1
  polarization_tab_whites[i,] <- NA
  polarization_tab_whites$topic_6[i] <- 'Total'
  polarization_tab_whites$party[i] <- p
  
  temp.pol <- get(paste0(polarization_tab_whites$party[i], '_polarization_boot_whites'))
  
  polarization_tab_whites$avgpol[i] <- median(colMeans(temp.pol))
  
  for (v in c('signif', 'gilens', 'gilenssignif', 'disagree', 'disagreegilens', 'n_quest')) { 
    polarization_tab_whites[i,v] <- sum(polarization_tab_whites[polarization_tab_whites$party == p, v], na.rm = TRUE)
  }
}

polarization_tab_whites %>% 
  select(party, topic_6, n_quest, avgpol, signif, gilens, disagreegilens) %>% 
  arrange(party) %>%
  mutate(party = case_when(party == 'rep' ~ 'Republicans',
                           party == 'dem' ~ 'Democrats',
                           party == 'all' ~ 'All Respondents'),
         topic_6 = case_when(topic_6 == 'foreignpolicy' ~ 'Foreign policy',
                             topic_6 == 'cultural' ~ 'Cultural',
                             topic_6 == 'immigration' ~ 'Immigration',
                             topic_6 == 'lawenforcement' ~ 'Law enforcement',
                             topic_6 == 'economic' ~ 'Economic',
                             topic_6 == 'socialwelfare' ~ 'Social welfare',
                             TRUE ~ topic_6)) %>% 
  mutate_at(vars(signif:disagreegilens), funs(. / n_quest)) %>% 
  mutate_if(is.numeric, round, 3) %>% 
  mutate_at(vars(signif:disagreegilens), funs(paste0(. * 100, '\\%'))) %>% 
  mutate(party = cell_spec(party, angle = 90, format = 'latex')) %>% 
  kable('latex', booktabs = TRUE, escape = FALSE, linesep = '\\addlinespace',
        col.names = linebreak(c(' ', 'Policy\nDomain', 'Number of\nUnique Issues', 'Average\nOpinion Gap', 'Opinion Difference\nStatistically\nSignificant', 
                                'Opinion Gap\nGreater than\n10 pts.', '10 pt. Gap +\nDisagreement\non Outcome'),
                              align = 'c'),
        align = c('l', 'l', rep('c', ncol(.)-2))) %>%
  collapse_rows(1, latex_hline = 'major') %>% 
  row_spec(c(7, 14, 21), bold = TRUE) %>% 
  save_kable(file = 'appendix_tables/detailed_polarization_summary_whites.tex')



# figure A9 - thresholds for white respondents

thresholds_df_whites <- expand.grid(question = issuetopics_whites$question,
                                    party = c('rep', 'dem', 'all'),
                                    threshold = (0:50)/100,
                                    polarized = NA,
                                    stringsAsFactors = FALSE)

for (i in 1:nrow(thresholds_df_whites)) {
  if (i %% floor(nrow(thresholds_df_whites)/10) == 0) cat('|')
  temp.rich <- get(paste0(thresholds_df_whites$party[i], '_inctop_10_whites'))
  temp.rich <- temp.rich[thresholds_df_whites$question[i],]
  temp.poor <- get(paste0(thresholds_df_whites$party[i], '_incbottom_10_whites'))
  temp.poor <- temp.poor[thresholds_df_whites$question[i],]
  
  t <- thresholds_df_whites$threshold[i]
  
  if (median(abs(temp.rich - temp.poor)) >= t) { 
    thresholds_df_whites$polarized[i] <- 1
  } else {
    thresholds_df_whites$polarized[i] <- 0
  }
}


thresholds_df_whites <- thresholds_df_whites %>% 
  mutate(party = case_when(party == 'dem' ~ 'Democrats',
                           party == 'rep' ~ 'Republicans',
                           party == 'all' ~ 'All Respondents')) %>% 
  group_by(party, threshold) %>% 
  summarize(n_pol = sum(polarized)) %>% 
  ungroup() %>% 
  filter(threshold <= 0.35)

ggplot(data = thresholds_df_whites,
       aes(x = threshold, y = n_pol / nrow(issuetopics_whites))) + 
  geom_point(aes(color = party, shape = party), size = 2, alpha = 0.4) + 
  geom_line(aes(color = party), size = 0.75, alpha = 0.5) + 
  labs(title = '',
       y = 'Share of Questions With Opinion Gap',
       x = 'Threshold',
       color = '', shape = '') +
  scale_color_manual(values = c('gray40', 'blue', 'red')) +
  theme_bw() +
  theme(legend.position = 'bottom')
ggsave('appendix_figures/num_polarized_by_threshold_whites.png', height = 4, width = 5.5)
